package no.ntnu.idi.deid.preprocessor.tokenizer;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;

import no.ntnu.idi.deid.config.Constants;
import no.ntnu.idi.deid.documentunits.Token;
import no.ntnu.idi.deid.preprocessor.xmlgenerator.XMLTokenGenerator;

public class Tokenizer {

	private ArrayList<Token> tokenList;
	private String inputFileName;
	private String outputFileName;

	public Tokenizer(String inFileName, boolean pos) throws IOException {
		inputFileName = inFileName;
		tokenList = new ArrayList<Token>();
		if(pos){posTokenize();}
		else{tokenize();}
	}
	
	public void printTokens() {
		for (Token word : tokenList) {
			System.out.println(word);
		}
	}
	
	public static ArrayList<Token> tokenizeTaggedFile(File file) throws IOException {
		Tokenizer t = new Tokenizer(file.getPath(), true);
		return t.getTokens();
	}

	public ArrayList<Token> getTokens() {
		return tokenList;
	}

	public void tokenize() throws IOException {
		Reader reader = new FileReader(inputFileName);

		BufferedReader br = new BufferedReader(reader);
		String line = br.readLine();
		int indexCounter = 0;
		while (line != null) {
			{
				String words[] = line.split(" ");
				for (int i = 0; i < words.length; i++) {
					if (!words[i].isEmpty()
							&& !words[i].contains(Constants.SECTION_SPLITTER)) {
						Token temp = new Token(indexCounter, words[i]);
						tokenList.add(temp);
						indexCounter++;
					}
				}
				line = br.readLine();
			}
		}
		br.close();
	}
	
	public void posTokenize() throws IOException {
		Reader reader = new FileReader(inputFileName);

		BufferedReader br = new BufferedReader(reader);
		String line = br.readLine();
		int indexCounter = 0;
		while (line != null) {
			{
				String words[] = line.split(" ");
				for (int i = 0; i < words.length; i++) {
					if (!words[i].isEmpty()
							&& !words[i].contains(Constants.SECTION_SPLITTER)) {
						Token temp = new Token(indexCounter, words[i], true);
						tokenList.add(temp);
						indexCounter++;
					}
				}
				line = br.readLine();
			}
		}
		br.close();
	}

	public void writeToXMLFile() {
		XMLTokenGenerator.generate(tokenList, outputFileName);
	}

}
